import json
import csv
import argparse
import os
from collections import Counter

import nltk

annotations = {
    "MS COCO": ["train_caps.txt", "dev_caps.txt"],
    "Flickr 30k": ["train_caps.txt", "dev_caps.txt"],
    "CC152K": ["train_caps.tsv", "dev_caps.tsv"],
}


class Vocabulary(object):
    """Simple vocabulary wrapper."""

    def __init__(self):
        self.word2idx = {}
        self.idx2word = {}
        self.idx = 0

    def __call__(self, word):
        if word not in self.word2idx:
            return self.word2idx["<unk>"]
        return self.word2idx[word]

    def add_word(self, word):
        if word not in self.word2idx:
            self.word2idx[word] = self.idx
            self.idx2word[self.idx] = word
            self.idx += 1

    def __len__(self):
        return len(self.word2idx)


def deserialize_vocab(src):
    with open(src) as f:
        d = json.load(f)
    vocab = Vocabulary()
    vocab.word2idx = d["word2idx"]
    vocab.idx2word = d["idx2word"]
    vocab.idx = d["idx"]
    return vocab


def serialize_vocab(vocab, dest):
    d = {"word2idx": vocab.word2idx, "idx2word": vocab.idx2word, "idx": vocab.idx}
    with open(dest, "w") as f:
        json.dump(d, f)


def build_vocab(data_path, data_name, caption_file, threshold):
    counter = Counter()
    for path in caption_file[data_name]:
        if data_name == "CC152K":
            full_path = os.path.join(data_path, data_name, path)
            # captions = from_tsv(full_path)
        elif data_name in ["MS COCO", "Flickr 30k"]:
            full_path = os.path.join(data_path, data_name, path)
            captions = from_txt(full_path)
        else:
            raise NotImplementedError("Not support!")
        for i, caption in enumerate(captions):
            tokens = nltk.tokenize.word_tokenize(caption.lower())
            counter.update(tokens)
            if i % 1000 == 0:
                print("[%d/%d] tokenized the captions." % (i, len(captions)))
    words = [word for word, cnt in counter.items() if cnt >= threshold]
    vocab = Vocabulary()
    vocab.add_word("<pad>")
    vocab.add_word("<start>")
    vocab.add_word("<end>")
    vocab.add_word("<unk>")

    # Add words to the vocabulary.
    for i, word in enumerate(words):
        vocab.add_word(word)
    return vocab


def from_txt(txt):
    captions = []
    with open(txt, "r", encoding='utf-8') as f:
        for line in f:
            captions.append(line.strip())
    return captions


def main():
    pass


if __name__ == '__main__':
    main()
